{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# load CSV file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: '/home/hll/IDS/CICIDS2017/CICIDS2017dataset/Wednesday-workingHours.pcap_ISCX.csv'",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mFileNotFoundError\u001B[0m                         Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[7], line 8\u001B[0m\n\u001B[0;32m      1\u001B[0m \u001B[38;5;66;03m# data = pd.read_csv('/home/hll/IDS/CICIDS2017/CICIDS2017dataset/Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv')\u001B[39;00m\n\u001B[0;32m      2\u001B[0m \u001B[38;5;66;03m# data = pd.read_csv('/home/hll/IDS/CICIDS2017/CICIDS2017dataset/Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX.csv')\u001B[39;00m\n\u001B[0;32m      3\u001B[0m \u001B[38;5;66;03m# data = pd.read_csv('/home/hll/IDS/CICIDS2017/CICIDS2017dataset/Friday-WorkingHours-Morning.pcap_ISCX.csv')\u001B[39;00m\n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m      6\u001B[0m \u001B[38;5;66;03m# data = pd.read_csv('/home/hll/IDS/CICIDS2017/CICIDS2017dataset/Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv')\u001B[39;00m\n\u001B[0;32m      7\u001B[0m \u001B[38;5;66;03m# data = pd.read_csv('/home/hll/IDS/CICIDS2017/CICIDS2017dataset/Tuesday-WorkingHours.pcap_ISCX.csv')\u001B[39;00m\n\u001B[1;32m----> 8\u001B[0m data \u001B[38;5;241m=\u001B[39m pd\u001B[38;5;241m.\u001B[39mread_csv(\u001B[38;5;124m'\u001B[39m\u001B[38;5;124m/home/hll/IDS/CICIDS2017/CICIDS2017dataset/Wednesday-workingHours.pcap_ISCX.csv\u001B[39m\u001B[38;5;124m'\u001B[39m)\n",
      "File \u001B[1;32mD:\\Anaconda\\Lib\\site-packages\\pandas\\io\\parsers\\readers.py:1026\u001B[0m, in \u001B[0;36mread_csv\u001B[1;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, date_format, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options, dtype_backend)\u001B[0m\n\u001B[0;32m   1013\u001B[0m kwds_defaults \u001B[38;5;241m=\u001B[39m _refine_defaults_read(\n\u001B[0;32m   1014\u001B[0m     dialect,\n\u001B[0;32m   1015\u001B[0m     delimiter,\n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m   1022\u001B[0m     dtype_backend\u001B[38;5;241m=\u001B[39mdtype_backend,\n\u001B[0;32m   1023\u001B[0m )\n\u001B[0;32m   1024\u001B[0m kwds\u001B[38;5;241m.\u001B[39mupdate(kwds_defaults)\n\u001B[1;32m-> 1026\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m _read(filepath_or_buffer, kwds)\n",
      "File \u001B[1;32mD:\\Anaconda\\Lib\\site-packages\\pandas\\io\\parsers\\readers.py:620\u001B[0m, in \u001B[0;36m_read\u001B[1;34m(filepath_or_buffer, kwds)\u001B[0m\n\u001B[0;32m    617\u001B[0m _validate_names(kwds\u001B[38;5;241m.\u001B[39mget(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mnames\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;28;01mNone\u001B[39;00m))\n\u001B[0;32m    619\u001B[0m \u001B[38;5;66;03m# Create the parser.\u001B[39;00m\n\u001B[1;32m--> 620\u001B[0m parser \u001B[38;5;241m=\u001B[39m TextFileReader(filepath_or_buffer, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwds)\n\u001B[0;32m    622\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m chunksize \u001B[38;5;129;01mor\u001B[39;00m iterator:\n\u001B[0;32m    623\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m parser\n",
      "File \u001B[1;32mD:\\Anaconda\\Lib\\site-packages\\pandas\\io\\parsers\\readers.py:1620\u001B[0m, in \u001B[0;36mTextFileReader.__init__\u001B[1;34m(self, f, engine, **kwds)\u001B[0m\n\u001B[0;32m   1617\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39moptions[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mhas_index_names\u001B[39m\u001B[38;5;124m\"\u001B[39m] \u001B[38;5;241m=\u001B[39m kwds[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mhas_index_names\u001B[39m\u001B[38;5;124m\"\u001B[39m]\n\u001B[0;32m   1619\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mhandles: IOHandles \u001B[38;5;241m|\u001B[39m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mNone\u001B[39;00m\n\u001B[1;32m-> 1620\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_engine \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_make_engine(f, \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mengine)\n",
      "File \u001B[1;32mD:\\Anaconda\\Lib\\site-packages\\pandas\\io\\parsers\\readers.py:1880\u001B[0m, in \u001B[0;36mTextFileReader._make_engine\u001B[1;34m(self, f, engine)\u001B[0m\n\u001B[0;32m   1878\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mb\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;129;01min\u001B[39;00m mode:\n\u001B[0;32m   1879\u001B[0m         mode \u001B[38;5;241m+\u001B[39m\u001B[38;5;241m=\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mb\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m-> 1880\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mhandles \u001B[38;5;241m=\u001B[39m get_handle(\n\u001B[0;32m   1881\u001B[0m     f,\n\u001B[0;32m   1882\u001B[0m     mode,\n\u001B[0;32m   1883\u001B[0m     encoding\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39moptions\u001B[38;5;241m.\u001B[39mget(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mencoding\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;28;01mNone\u001B[39;00m),\n\u001B[0;32m   1884\u001B[0m     compression\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39moptions\u001B[38;5;241m.\u001B[39mget(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mcompression\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;28;01mNone\u001B[39;00m),\n\u001B[0;32m   1885\u001B[0m     memory_map\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39moptions\u001B[38;5;241m.\u001B[39mget(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mmemory_map\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;28;01mFalse\u001B[39;00m),\n\u001B[0;32m   1886\u001B[0m     is_text\u001B[38;5;241m=\u001B[39mis_text,\n\u001B[0;32m   1887\u001B[0m     errors\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39moptions\u001B[38;5;241m.\u001B[39mget(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mencoding_errors\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mstrict\u001B[39m\u001B[38;5;124m\"\u001B[39m),\n\u001B[0;32m   1888\u001B[0m     storage_options\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39moptions\u001B[38;5;241m.\u001B[39mget(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mstorage_options\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;28;01mNone\u001B[39;00m),\n\u001B[0;32m   1889\u001B[0m )\n\u001B[0;32m   1890\u001B[0m \u001B[38;5;28;01massert\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mhandles \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n\u001B[0;32m   1891\u001B[0m f \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mhandles\u001B[38;5;241m.\u001B[39mhandle\n",
      "File \u001B[1;32mD:\\Anaconda\\Lib\\site-packages\\pandas\\io\\common.py:873\u001B[0m, in \u001B[0;36mget_handle\u001B[1;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001B[0m\n\u001B[0;32m    868\u001B[0m \u001B[38;5;28;01melif\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(handle, \u001B[38;5;28mstr\u001B[39m):\n\u001B[0;32m    869\u001B[0m     \u001B[38;5;66;03m# Check whether the filename is to be opened in binary mode.\u001B[39;00m\n\u001B[0;32m    870\u001B[0m     \u001B[38;5;66;03m# Binary mode does not support 'encoding' and 'newline'.\u001B[39;00m\n\u001B[0;32m    871\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m ioargs\u001B[38;5;241m.\u001B[39mencoding \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mb\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;129;01min\u001B[39;00m ioargs\u001B[38;5;241m.\u001B[39mmode:\n\u001B[0;32m    872\u001B[0m         \u001B[38;5;66;03m# Encoding\u001B[39;00m\n\u001B[1;32m--> 873\u001B[0m         handle \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mopen\u001B[39m(\n\u001B[0;32m    874\u001B[0m             handle,\n\u001B[0;32m    875\u001B[0m             ioargs\u001B[38;5;241m.\u001B[39mmode,\n\u001B[0;32m    876\u001B[0m             encoding\u001B[38;5;241m=\u001B[39mioargs\u001B[38;5;241m.\u001B[39mencoding,\n\u001B[0;32m    877\u001B[0m             errors\u001B[38;5;241m=\u001B[39merrors,\n\u001B[0;32m    878\u001B[0m             newline\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[0;32m    879\u001B[0m         )\n\u001B[0;32m    880\u001B[0m     \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m    881\u001B[0m         \u001B[38;5;66;03m# Binary mode\u001B[39;00m\n\u001B[0;32m    882\u001B[0m         handle \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mopen\u001B[39m(handle, ioargs\u001B[38;5;241m.\u001B[39mmode)\n",
      "\u001B[1;31mFileNotFoundError\u001B[0m: [Errno 2] No such file or directory: '/home/hll/IDS/CICIDS2017/CICIDS2017dataset/Wednesday-workingHours.pcap_ISCX.csv'"
     ]
    }
   ],
   "source": [
    "# data = pd.read_csv('/home/hll/IDS/CICIDS2017/CICIDS2017dataset/Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv')\n",
    "# data = pd.read_csv('/home/hll/IDS/CICIDS2017/CICIDS2017dataset/Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX.csv')\n",
    "# data = pd.read_csv('/home/hll/IDS/CICIDS2017/CICIDS2017dataset/Friday-WorkingHours-Morning.pcap_ISCX.csv')\n",
    "# data = pd.read_csv('/home/hll/IDS/CICIDS2017/CICIDS2017dataset/Monday-WorkingHours.pcap_ISCX.csv')\n",
    "# data = pd.read_csv('/home/hll/IDS/CICIDS2017/CICIDS2017dataset/Thursday-WorkingHours-Afternoon-Infilteration.pcap_ISCX.csv')\n",
    "# data = pd.read_csv('/home/hll/IDS/CICIDS2017/CICIDS2017dataset/Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv')\n",
    "# data = pd.read_csv('/home/hll/IDS/CICIDS2017/CICIDS2017dataset/Tuesday-WorkingHours.pcap_ISCX.csv')\n",
    "data = pd.read_csv('/home/hll/IDS/CICIDS2017/CICIDS2017dataset/Wednesday-workingHours.pcap_ISCX.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Destination Port</th>\n",
       "      <th>Flow Duration</th>\n",
       "      <th>Total Fwd Packets</th>\n",
       "      <th>Total Backward Packets</th>\n",
       "      <th>Total Length of Fwd Packets</th>\n",
       "      <th>Total Length of Bwd Packets</th>\n",
       "      <th>Fwd Packet Length Max</th>\n",
       "      <th>Fwd Packet Length Min</th>\n",
       "      <th>Fwd Packet Length Mean</th>\n",
       "      <th>Fwd Packet Length Std</th>\n",
       "      <th>...</th>\n",
       "      <th>min_seg_size_forward</th>\n",
       "      <th>Active Mean</th>\n",
       "      <th>Active Std</th>\n",
       "      <th>Active Max</th>\n",
       "      <th>Active Min</th>\n",
       "      <th>Idle Mean</th>\n",
       "      <th>Idle Std</th>\n",
       "      <th>Idle Max</th>\n",
       "      <th>Idle Min</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>80</td>\n",
       "      <td>38308</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>BENIGN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>389</td>\n",
       "      <td>479</td>\n",
       "      <td>11</td>\n",
       "      <td>5</td>\n",
       "      <td>172</td>\n",
       "      <td>326</td>\n",
       "      <td>79</td>\n",
       "      <td>0</td>\n",
       "      <td>15.636364</td>\n",
       "      <td>31.449238</td>\n",
       "      <td>...</td>\n",
       "      <td>32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>BENIGN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>88</td>\n",
       "      <td>1095</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>3150</td>\n",
       "      <td>3150</td>\n",
       "      <td>1575</td>\n",
       "      <td>0</td>\n",
       "      <td>315.000000</td>\n",
       "      <td>632.561635</td>\n",
       "      <td>...</td>\n",
       "      <td>32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>BENIGN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>389</td>\n",
       "      <td>15206</td>\n",
       "      <td>17</td>\n",
       "      <td>12</td>\n",
       "      <td>3452</td>\n",
       "      <td>6660</td>\n",
       "      <td>1313</td>\n",
       "      <td>0</td>\n",
       "      <td>203.058824</td>\n",
       "      <td>425.778474</td>\n",
       "      <td>...</td>\n",
       "      <td>32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>BENIGN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>88</td>\n",
       "      <td>1092</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>3150</td>\n",
       "      <td>3152</td>\n",
       "      <td>1575</td>\n",
       "      <td>0</td>\n",
       "      <td>350.000000</td>\n",
       "      <td>694.509719</td>\n",
       "      <td>...</td>\n",
       "      <td>32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>BENIGN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>692698</td>\n",
       "      <td>53</td>\n",
       "      <td>32215</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>112</td>\n",
       "      <td>152</td>\n",
       "      <td>28</td>\n",
       "      <td>28</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>BENIGN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>692699</td>\n",
       "      <td>53</td>\n",
       "      <td>324</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>84</td>\n",
       "      <td>362</td>\n",
       "      <td>42</td>\n",
       "      <td>42</td>\n",
       "      <td>42.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>BENIGN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>692700</td>\n",
       "      <td>58030</td>\n",
       "      <td>82</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>31</td>\n",
       "      <td>6</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "      <td>15.500000</td>\n",
       "      <td>21.920310</td>\n",
       "      <td>...</td>\n",
       "      <td>32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>BENIGN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>692701</td>\n",
       "      <td>53</td>\n",
       "      <td>1048635</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>192</td>\n",
       "      <td>256</td>\n",
       "      <td>32</td>\n",
       "      <td>32</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>BENIGN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>692702</td>\n",
       "      <td>53</td>\n",
       "      <td>94939</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>188</td>\n",
       "      <td>226</td>\n",
       "      <td>47</td>\n",
       "      <td>47</td>\n",
       "      <td>47.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>BENIGN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>692703 rows × 78 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Destination Port   Flow Duration   Total Fwd Packets  \\\n",
       "0                      80           38308                   1   \n",
       "1                     389             479                  11   \n",
       "2                      88            1095                  10   \n",
       "3                     389           15206                  17   \n",
       "4                      88            1092                   9   \n",
       "...                   ...             ...                 ...   \n",
       "692698                 53           32215                   4   \n",
       "692699                 53             324                   2   \n",
       "692700              58030              82                   2   \n",
       "692701                 53         1048635                   6   \n",
       "692702                 53           94939                   4   \n",
       "\n",
       "         Total Backward Packets  Total Length of Fwd Packets  \\\n",
       "0                             1                            6   \n",
       "1                             5                          172   \n",
       "2                             6                         3150   \n",
       "3                            12                         3452   \n",
       "4                             6                         3150   \n",
       "...                         ...                          ...   \n",
       "692698                        2                          112   \n",
       "692699                        2                           84   \n",
       "692700                        1                           31   \n",
       "692701                        2                          192   \n",
       "692702                        2                          188   \n",
       "\n",
       "         Total Length of Bwd Packets   Fwd Packet Length Max  \\\n",
       "0                                  6                       6   \n",
       "1                                326                      79   \n",
       "2                               3150                    1575   \n",
       "3                               6660                    1313   \n",
       "4                               3152                    1575   \n",
       "...                              ...                     ...   \n",
       "692698                           152                      28   \n",
       "692699                           362                      42   \n",
       "692700                             6                      31   \n",
       "692701                           256                      32   \n",
       "692702                           226                      47   \n",
       "\n",
       "         Fwd Packet Length Min   Fwd Packet Length Mean  \\\n",
       "0                            6                 6.000000   \n",
       "1                            0                15.636364   \n",
       "2                            0               315.000000   \n",
       "3                            0               203.058824   \n",
       "4                            0               350.000000   \n",
       "...                        ...                      ...   \n",
       "692698                      28                28.000000   \n",
       "692699                      42                42.000000   \n",
       "692700                       0                15.500000   \n",
       "692701                      32                32.000000   \n",
       "692702                      47                47.000000   \n",
       "\n",
       "         Fwd Packet Length Std  ...   min_seg_size_forward  Active Mean  \\\n",
       "0                     0.000000  ...                     20          0.0   \n",
       "1                    31.449238  ...                     32          0.0   \n",
       "2                   632.561635  ...                     32          0.0   \n",
       "3                   425.778474  ...                     32          0.0   \n",
       "4                   694.509719  ...                     32          0.0   \n",
       "...                        ...  ...                    ...          ...   \n",
       "692698                0.000000  ...                     20          0.0   \n",
       "692699                0.000000  ...                     20          0.0   \n",
       "692700               21.920310  ...                     32          0.0   \n",
       "692701                0.000000  ...                     20          0.0   \n",
       "692702                0.000000  ...                     20          0.0   \n",
       "\n",
       "         Active Std   Active Max  Active Min Idle Mean   Idle Std   Idle Max  \\\n",
       "0               0.0            0           0       0.0        0.0          0   \n",
       "1               0.0            0           0       0.0        0.0          0   \n",
       "2               0.0            0           0       0.0        0.0          0   \n",
       "3               0.0            0           0       0.0        0.0          0   \n",
       "4               0.0            0           0       0.0        0.0          0   \n",
       "...             ...          ...         ...       ...        ...        ...   \n",
       "692698          0.0            0           0       0.0        0.0          0   \n",
       "692699          0.0            0           0       0.0        0.0          0   \n",
       "692700          0.0            0           0       0.0        0.0          0   \n",
       "692701          0.0            0           0       0.0        0.0          0   \n",
       "692702          0.0            0           0       0.0        0.0          0   \n",
       "\n",
       "         Idle Min   Label  \n",
       "0               0  BENIGN  \n",
       "1               0  BENIGN  \n",
       "2               0  BENIGN  \n",
       "3               0  BENIGN  \n",
       "4               0  BENIGN  \n",
       "...           ...     ...  \n",
       "692698          0  BENIGN  \n",
       "692699          0  BENIGN  \n",
       "692700          0  BENIGN  \n",
       "692701          0  BENIGN  \n",
       "692702          0  BENIGN  \n",
       "\n",
       "[692703 rows x 78 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([' Destination Port', ' Flow Duration', ' Total Fwd Packets',\n",
       "       ' Total Backward Packets', 'Total Length of Fwd Packets',\n",
       "       ' Total Length of Bwd Packets', ' Fwd Packet Length Max',\n",
       "       ' Fwd Packet Length Min', ' Fwd Packet Length Mean',\n",
       "       ' Fwd Packet Length Std', 'Bwd Packet Length Max',\n",
       "       ' Bwd Packet Length Min', ' Bwd Packet Length Mean',\n",
       "       ' Bwd Packet Length Std', 'Flow Bytes/s', ' Flow Packets/s',\n",
       "       ' Flow IAT Mean', ' Flow IAT Std', ' Flow IAT Max', ' Flow IAT Min',\n",
       "       'Fwd IAT Total', ' Fwd IAT Mean', ' Fwd IAT Std', ' Fwd IAT Max',\n",
       "       ' Fwd IAT Min', 'Bwd IAT Total', ' Bwd IAT Mean', ' Bwd IAT Std',\n",
       "       ' Bwd IAT Max', ' Bwd IAT Min', 'Fwd PSH Flags', ' Bwd PSH Flags',\n",
       "       ' Fwd URG Flags', ' Bwd URG Flags', ' Fwd Header Length',\n",
       "       ' Bwd Header Length', 'Fwd Packets/s', ' Bwd Packets/s',\n",
       "       ' Min Packet Length', ' Max Packet Length', ' Packet Length Mean',\n",
       "       ' Packet Length Std', ' Packet Length Variance', 'FIN Flag Count',\n",
       "       ' SYN Flag Count', ' RST Flag Count', ' PSH Flag Count',\n",
       "       ' ACK Flag Count', ' URG Flag Count', ' CWE Flag Count',\n",
       "       ' ECE Flag Count', ' Down/Up Ratio', ' Average Packet Size',\n",
       "       ' Avg Fwd Segment Size', ' Avg Bwd Segment Size', 'Fwd Avg Bytes/Bulk',\n",
       "       ' Fwd Avg Packets/Bulk', ' Fwd Avg Bulk Rate', ' Bwd Avg Bytes/Bulk',\n",
       "       ' Bwd Avg Packets/Bulk', 'Bwd Avg Bulk Rate', 'Subflow Fwd Packets',\n",
       "       ' Subflow Fwd Bytes', ' Subflow Bwd Packets', ' Subflow Bwd Bytes',\n",
       "       'Init_Win_bytes_forward', ' Init_Win_bytes_backward',\n",
       "       ' act_data_pkt_fwd', ' min_seg_size_forward', 'Active Mean',\n",
       "       ' Active Std', ' Active Max', ' Active Min', 'Idle Mean', ' Idle Std',\n",
       "       ' Idle Max', ' Idle Min', ' Label'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#第一行为特征名称\n",
    "#features of the name\n",
    "data.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#删除Timestamp特征\n",
    "#Delete the Timestamp feature\n",
    "data = data.drop(['Timestamp'],axis=1)\n",
    "\n",
    "# data = data.drop(['Flow ID','Src IP','Src Port','Dst IP'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(692703, 78)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "#缺失值（nan）补零\n",
    "#Zero the missing value (nan)\n",
    "data = data.fillna(value=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Replace the infinite value in the Flow Byts/s column with the maximum value +1 of its column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Flow Byts/s\n",
    "# pd.set_option('display.max_rows', 456531)\n",
    "# pd.value_counts(data['Flow Bytes/s'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hll/anaconda3/envs/tensorflow/lib/python3.6/site-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(data['Flow Bytes/s'])):\n",
    "    if data['Flow Bytes/s'][i]=='Infinity':\n",
    "        #print(i)\n",
    "        data['Flow Bytes/s'][i]= '1040000001'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pd.value_counts(data['Flow Bytes/s'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Replace the infinite value in the Flow Pkts/s column with the maximum value +1 of its column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Flow Pkts/s\n",
    "# pd.set_option('display.max_rows', 463132)\n",
    "# pd.value_counts(data[' Flow Packets/s'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hll/anaconda3/envs/tensorflow/lib/python3.6/site-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(data[' Flow Packets/s'])):\n",
    "    if data[' Flow Packets/s'][i]=='Infinity':\n",
    "        #print(i)\n",
    "        data[' Flow Packets/s'][i]= '2000001'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pd.value_counts(data[' Flow Packets/s'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BENIGN              440031\n",
       "DoS Hulk            231073\n",
       "DoS GoldenEye        10293\n",
       "DoS slowloris         5796\n",
       "DoS Slowhttptest      5499\n",
       "Heartbleed              11\n",
       "Name:  Label, dtype: int64"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.value_counts(data[' Label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #部分文件中出现了多个特征名行，需要将他们删除\n",
    "#Several feature name lines appear in some files and need to be deleted\n",
    "\n",
    "# for i in range(len(data[' Label'])):\n",
    "#     if data[' Label'][i]==' Label':\n",
    "#         #print(i)\n",
    "#         data.drop(i,axis=0,inplace=True)\n",
    "        \n",
    "# pd.value_counts(data[' Label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(692703, 78)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(692703, 77)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hll/anaconda3/envs/tensorflow/lib/python3.6/site-packages/ipykernel_launcher.py:4: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "#data\n",
    "data_ = data.drop([' Label'],axis=1)\n",
    "print(data_.shape)\n",
    "data_ed = data_.as_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "#StandardScaler\n",
    "from sklearn.preprocessing import  StandardScaler\n",
    "data_ed=StandardScaler().fit_transform(data_ed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Label numeralization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hll/anaconda3/envs/tensorflow/lib/python3.6/site-packages/ipykernel_launcher.py:3: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    }
   ],
   "source": [
    "L = data[' Label']\n",
    "\n",
    "L = L.as_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "#将15分类标签数值化\n",
    "#Numeric the 15 category label\n",
    "\n",
    "i=0\n",
    "for i in range(L.shape[0]):\n",
    "    if L[i] == \"BENIGN\":\n",
    "        L[i]=0\n",
    "    elif L[i] == \"DoS Hulk\":\n",
    "        L[i]=1\n",
    "    elif L[i] == \"PortScan\":\n",
    "        L[i]=2\n",
    "    elif L[i] == \"DDoS\":\n",
    "        L[i]=3\n",
    "    elif L[i] == \"DoS GoldenEye\":\n",
    "        L[i]=4\n",
    "    elif L[i] == \"FTP-Patator\":\n",
    "        L[i]=5\n",
    "    elif L[i] == \"SSH-Patator\":\n",
    "        L[i]=6\n",
    "    elif L[i] == \"DoS slowloris\":\n",
    "        L[i]=7\n",
    "    elif L[i] == \"DoS Slowhttptest\":\n",
    "        L[i]=8\n",
    "    elif L[i] == \"Bot\":\n",
    "        L[i]=9\n",
    "    elif L[i] == \"Web Attack Brute Force\":\n",
    "        L[i]=10\n",
    "    elif L[i] == \"Web Attack XSS\":\n",
    "        L[i]=11\n",
    "    elif L[i] == \"Infiltration\":\n",
    "        L[i]=12\n",
    "    elif L[i] == \"Web Attack Sql Injection\":\n",
    "        L[i]=13\n",
    "    elif L[i] == \"Heartbleed\":\n",
    "        L[i]=14\n",
    "        \n",
    "        i=i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 440031), (1, 231073), (4, 10293), (7, 5796), (8, 5499), (14, 11)]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#查看数据类分布\n",
    "#View the data class distribution\n",
    "from collections import Counter\n",
    "sorted(Counter(L).items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = L.reshape(L.shape[0],1)  \n",
    "\n",
    "np.save(\"/home/hll/IDS/CICIDS2017/data/data_8.npy\",data_ed)  \n",
    "np.save(\"/home/hll/IDS/CICIDS2017/data/label_8.npy\",label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Integrate data from 8 CSV files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "data1 = np.load('/home/hll/IDS/CICIDS2017/data/data_1.npy')\n",
    "data2 = np.load('/home/hll/IDS/CICIDS2017/data/data_2.npy')\n",
    "data3 = np.load('/home/hll/IDS/CICIDS2017/data/data_3.npy')\n",
    "data4 = np.load('/home/hll/IDS/CICIDS2017/data/data_4.npy')\n",
    "data5 = np.load('/home/hll/IDS/CICIDS2017/data/data_5.npy')\n",
    "data6 = np.load('/home/hll/IDS/CICIDS2017/data/data_6.npy')\n",
    "data7 = np.load('/home/hll/IDS/CICIDS2017/data/data_7.npy')\n",
    "data8 = np.load('/home/hll/IDS/CICIDS2017/data/data_8.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(225745, 77)\n",
      "(286467, 77)\n",
      "(191033, 77)\n",
      "(529918, 77)\n",
      "(288602, 77)\n",
      "(170366, 77)\n",
      "(445909, 77)\n",
      "(692703, 77)\n"
     ]
    }
   ],
   "source": [
    "print(data1.shape)\n",
    "print(data2.shape)\n",
    "print(data3.shape)\n",
    "print(data4.shape)\n",
    "print(data5.shape)\n",
    "print(data6.shape)\n",
    "print(data7.shape)\n",
    "print(data8.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2830743, 77)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = np.concatenate((data1,data2,data3,data4,data5,data6,data7,data8),axis=0)\n",
    "\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "label1 = np.load('/home/hll/IDS/CICIDS2017/data/label_1.npy')\n",
    "label2 = np.load('/home/hll/IDS/CICIDS2017/data/label_2.npy')\n",
    "label3 = np.load('/home/hll/IDS/CICIDS2017/data/label_3.npy')\n",
    "label4 = np.load('/home/hll/IDS/CICIDS2017/data/label_4.npy')\n",
    "label5 = np.load('/home/hll/IDS/CICIDS2017/data/label_5.npy')\n",
    "label6 = np.load('/home/hll/IDS/CICIDS2017/data/label_6.npy')\n",
    "label7 = np.load('/home/hll/IDS/CICIDS2017/data/label_7.npy')\n",
    "label8 = np.load('/home/hll/IDS/CICIDS2017/data/label_8.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(225745, 1)\n",
      "(286467, 1)\n",
      "(191033, 1)\n",
      "(529918, 1)\n",
      "(288602, 1)\n",
      "(170366, 1)\n",
      "(445909, 1)\n",
      "(692703, 1)\n"
     ]
    }
   ],
   "source": [
    "print(label1.shape)\n",
    "print(label2.shape)\n",
    "print(label3.shape)\n",
    "print(label4.shape)\n",
    "print(label5.shape)\n",
    "print(label6.shape)\n",
    "print(label7.shape)\n",
    "print(label8.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2830743, 1)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label = np.concatenate((label1,label2,label3,label4,label5,label6,label7,label8),axis=0)\n",
    "\n",
    "label.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"/home/hll/IDS/CICIDS2017/data/data.npy\",data) \n",
    "np.save(\"/home/hll/IDS/CICIDS2017/data/label.npy\",label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 2273097),\n",
       " (1, 231073),\n",
       " (2, 158930),\n",
       " (3, 128027),\n",
       " (4, 10293),\n",
       " (5, 7938),\n",
       " (6, 5897),\n",
       " (7, 5796),\n",
       " (8, 5499),\n",
       " (9, 1966),\n",
       " (10, 1507),\n",
       " (11, 652),\n",
       " (12, 36),\n",
       " (13, 21),\n",
       " (14, 11)]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b15=np.array(label) \n",
    "bb15=b15.reshape(b15.shape[0],)#reshape重新定义形状,此时标签数据是1维的\n",
    "y15 = np.int32(bb15)\n",
    "\n",
    "sorted(Counter(y15).items())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# train:val:test=7:1:2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "训练集数据大小： (1981519, 77)\n",
      "训练集标签大小： (1981519, 1)\n",
      "验证集数据大小： (283075, 77)\n",
      "验证集标签大小： (283075, 1)\n",
      "测试集数据大小： (566149, 77)\n",
      "测试集标签大小： (566149, 1)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "x_train,x_test,y_train,y_test = train_test_split(data,label,test_size=0.2,random_state=0,stratify=label)\n",
    "x_train,x_val,y_train,y_val = train_test_split(x_train,y_train,test_size=0.125,random_state=0,stratify=y_train)\n",
    "\n",
    "print(\"训练集数据大小：\",x_train.shape)\n",
    "print(\"训练集标签大小：\",y_train.shape)\n",
    "print(\"验证集数据大小：\",x_val.shape)\n",
    "print(\"验证集标签大小：\",y_val.shape)\n",
    "print(\"测试集数据大小：\",x_test.shape)\n",
    "print(\"测试集标签大小：\",y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "b15=np.array(y_val)\n",
    "bb15=b15.reshape(b15.shape[0],)\n",
    "y15 = np.int32(bb15)\n",
    "\n",
    "sorted(Counter(y15).items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"/home/hll/IDS/CICIDS2017/data/data_train.npy\",x_train)\n",
    "np.save(\"/home/hll/IDS/CICIDS2017/data/label_train.npy\",y_train)\n",
    "np.save(\"/home/hll/IDS/CICIDS2017/data/data_test.npy\",x_test)\n",
    "np.save(\"/home/hll/IDS/CICIDS2017/data/label_test.npy\",y_test)\n",
    "np.save(\"/home/hll/IDS/CICIDS2017/data/data_val.npy\",x_val)\n",
    "np.save(\"/home/hll/IDS/CICIDS2017/data/label_val.npy\",y_val)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env_name",
   "language": "python",
   "name": "env_name"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}